package com.sean;


import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.bson.codecs.configuration.CodecRegistry;
import org.bson.codecs.pojo.PojoCodecProvider;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;

/**
 * Created by ailly on 17-9-7.
 */
public class DataCleansing {
    //    private String dataFilePath = "/home/ailly/SeanLab/java_code/data/test/split_file00";
    private String dataFilePath;

    private String rootSavePath;
    private List<Article> articles;

    private long articleNum = 0;    //文章总数
    private long imgNum = 0;    //图片总数
    private long articleWithImgNum = 0; //带图文章数量
    private long imgWithSrcNum = 0;     //带src标签图片数量
    private long imgWithAltNum = 0;     //带alt标签图片数量
    private long imgWithNonEmptySrcNum = 0;     //src标签非空图片数量
    private long imgWithNonEmptyAltNum = 0;     //alt标签非空图片数量
    private long validImgNum = 0;       //有效图片数量（能下载下来）
    private long readCount = 0;
    public long tmpCount = 0;

    public void readArticleData() {
        readArticleData(dataFilePath, rootSavePath);
    }

    public void readArticleData(String dataFilePath, String rootSavePath) {
        this.dataFilePath = dataFilePath;
        this.rootSavePath = rootSavePath;
        articles = new ArrayList<Article>();
        if (dataFilePath != null) {
            File articleData = new File(dataFilePath);
            try {
                FileReader fr = new FileReader(articleData);
                BufferedReader br = new BufferedReader(fr);

                String singleArticle = null;
                Article tmpArticle;

                while ( (singleArticle = br.readLine()) != null) {
                    String[] s = null;
                    try {
                        readCount++;
                        s = singleArticle.split("\t");

                        if (s.length == 7) {
                            tmpArticle = new Article();
                            //设置文章字段
                            tmpArticle.setArticleID(UUID.randomUUID().toString());
                            tmpArticle.setType(extractType(s[0]));
                            tmpArticle.setTags(extractTags(s[1]));
                            tmpArticle.setSource(s[2]);
                            tmpArticle.setChannel(s[3]);
                            tmpArticle.setUrl(s[4]);
                            tmpArticle.setContent(extractContent(s[5]));

                            //设置图片相关字段
                            tmpArticle.setImgID(extractImgID(s[5]));
                            tmpArticle.setImgURL(extractImgURL(s[5]));
                            tmpArticle.setImgOriginalURL(extractImgOriginalURL(s[5]));
                            tmpArticle.setImgDescription(extractImgDescription(s[5]));

                            this.articles.add(tmpArticle);
                        } else {
                            System.out.println("错误：数据格式异常！数据行号：" + readCount);
                        }


//                    System.out.println(tmpArticle.getArticleID());
//                    System.out.println(tmpArticle.getTags());
//                    System.out.println(tmpArticle.getType());
//                    System.out.println(tmpArticle.getSource());
//                    System.out.println(tmpArticle.getChannel());
//                    System.out.println(tmpArticle.getUrl());
//                    System.out.println(tmpArticle.getContent());
//
//                    System.out.println(tmpArticle.getImgID());
//                    System.out.println(tmpArticle.getImgURL());
//                    System.out.println(tmpArticle.getImgOriginalURL());
//                    System.out.println(tmpArticle.getImgDescription());
                    } catch (Exception e) {
                        e.printStackTrace();
                        //System.out.println(singleArticle + " " + readCount);
                    }
                }

            } catch (Exception e) {
                e.printStackTrace();
            }

        }

    }

    private void insertDoc(Article article) {

    }

    List<String> extractTags(String tagStr) {
        List<String> tmpTagList = new ArrayList<String>();
        String TagsPatttern = "\\\"(([\\u4e00-\\u9fa5])*?)\\\"";
        Matcher m = Pattern.compile(TagsPatttern).matcher(tagStr);
        int i = 0;
        while (m.find(i)) {
            tmpTagList.add(m.group(1));
            i = m.end();
        }
        return tmpTagList;
    }

    String extractType(String typeStr) {
        String categoryPattern = "\\[\\\"(\\w*?)\\\"\\]";
        Matcher m = Pattern.compile(categoryPattern).matcher(typeStr);
        if (m.find(0)) {
            return m.group(1);
        } else {
            return null;
        }
    }

    String extractContent(String contentStr) {
        Document article = Jsoup.parse(contentStr);

        String content = article.body().text();
        if (content == null || content == "") {
            System.out.println("文章内容读取出错" + contentStr);
            return "";
        } else {
            this.articleNum++;
            return content;
        }
    }

    List<String> extractImgID(String contentStr) {
        List<String> imgIDList = new ArrayList<String>();
        Document article = Jsoup.parse(contentStr);
        Elements img = article.select("img");
        String t = null;
        for (Element s : img) {
            try {
                if (s.hasAttr("src")) {
                    t = s.attr("src");
                    if (t.charAt(0) == '\\') {      //处理元数据中imgURL的转义引号字符
                        //取得图片原始url
                        if (t.split("\\\\\"").length == 0) {
                            imgIDList.add("");
                        } else {
                            String originID = t.split("\\\\\"")[1];
                            String[] tmp = originID.split("/");     //取得图片名字
                            String imgName = tmp[tmp.length - 1];
                            imgIDList.add( this.getMD5(imgName));     //计算保存路径
                        }
                    } else {
                        String originID = t;
                        if (originID == null || originID == "") {
                            imgIDList.add("");
                        } else {
                            String[] tmp = originID.split("/");     //取得图片名字
                            String imgName = tmp[tmp.length - 1];
                            imgIDList.add( this.getMD5(imgName) );     //计算保存路径
                        }
                    }
                } else {    //没有src属性时，添加空字符串
                    imgIDList.add("");
                }
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println("Error in extractImgID: img_info: " + s.toString() + " img.src: " +  s.attr("src") );
            }
        }
        return imgIDList;
    }

    List<String> extractImgURL(String contentStr) {
        List<String> imgURLList = new ArrayList<String>();
        Document article = Jsoup.parse(contentStr);
        Elements img = article.select("img");

        for (Element s : img) {
            //统计带src属性img数量
            if (s.hasAttr("src")) {
                String t = s.attr("src");
                if (t.charAt(0) == '\\') {      //处理元数据中imgURL的转义引号字符
                    if (t.split("\\\\\"").length == 0) {
                        imgURLList.add("");
                    } else {
                        String originURL = t.split("\\\\\"")[1];    //取得图片原始url
                        String[] tmp = originURL.split("/");     //取得图片名字
                        String imgName = tmp[tmp.length - 1];
                        try {
                            String tmpImgName = getSavePath(imgName);
                            File tmpFile = new File(tmpImgName);

                            // System.out.println(tmpImgName);

                            if (tmpFile.exists()) {
                                validImgNum++;
                                imgURLList.add(getSavePath(imgName));     //计算保存路径
                            } else {
                                imgURLList.add("");
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                            System.out.println("图片名格式错误: " + imgName + " " + t);
                        }
                    }
                } else {
                    String originURL = t;
                    if (originURL == null || originURL == "") {
                        imgURLList.add("");
                    } else {
                        String[] tmp = originURL.split("/");     //取得图片名字
                        String imgName = tmp[tmp.length - 1];
                        try {
                            String tmpImgName = getSavePath(imgName);
                            File tmpFile = new File(tmpImgName);
                            if (tmpFile.exists()) {
                                validImgNum++;
                                imgURLList.add(getSavePath(imgName));     //计算保存路径
                            } else {
                                imgURLList.add("");
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                            System.out.println("图片名格式错误: " + imgName);
                        }
                    }
                }
            } else {    //没有src属性时，添加空字符串
                imgURLList.add("");
            }
        }
        return imgURLList;
    }

    List<String> extractImgOriginalURL(String contentStr) {
        List<String> imgOriginalURLList = new ArrayList<String>();
        Document article = Jsoup.parse(contentStr);
        Elements img = article.select("img");

        //统计带图文章数 & 图片数(可能重复)
        if (img.size() != 0) {
            this.articleWithImgNum++;
            this.imgNum += img.size();
        }

        for (Element s : img) {
            //统计带src属性img数量
            if (s.hasAttr("src")) {
                imgWithSrcNum++;
                String t = s.attr("src");
                if (t.charAt(0) == '\\') {      //处理元数据中imgURL的转义引号字符
                    if (t.split("\\\\\"").length == 0) {
                        imgOriginalURLList.add("");
                    } else {
                        String imgOriginalURL = t.split("\\\\\"")[1];
                        imgOriginalURLList.add(imgOriginalURL);
                        imgWithNonEmptySrcNum++;
                    }
                } else {
                    String imgOriginalURL = t;
                    if (imgOriginalURL == null || imgOriginalURL == "") {
                        imgOriginalURLList.add("");
                    } else {
                        imgOriginalURLList.add( imgOriginalURL );     //计算保存路径
                        imgWithNonEmptySrcNum++;
                    }

                }
            } else {
                imgOriginalURLList.add("");
            }
        }
        return imgOriginalURLList;
    }

    List<String> extractImgDescription(String contentStr) {
        List<String> imgDescriptionList = new ArrayList<String>();
        Document article = Jsoup.parse(contentStr);
        Elements img = article.select("img");

        for (Element s : img) {
            //统计带alt属性img数量
            if (s.hasAttr("alt")) {
                imgWithAltNum++;
                String t = s.attr("alt");
                try {
                    if (t.charAt(0) == '\\') {      //处理元数据中imgDescription的转义引号字符
                        if(t.split("\\\\\"").length != 0) {
                            imgDescriptionList.add(t.split("\\\\\"")[1]);
                            imgWithNonEmptyAltNum++;
                        } else {
                            imgDescriptionList.add("");
                        }
                    } else {
                        if (t != "") {
                            imgDescriptionList.add(t);
                            imgWithNonEmptyAltNum++;
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    System.out.println("error in extractImgDescription: " + t);
                }
            } else {
                imgDescriptionList.add("");
            }
        }
        return imgDescriptionList;
    }

    //取得图片保存路径
    public String getSavePath(String imgName){
        if (rootSavePath == null || rootSavePath == "") {
            System.out.println("错误：未设置存储路径！");
            System.exit(1);
        }
        String newName = this.getMD5(imgName);
        //文章名存储格式
        if (imgName.split("\\.").length > 1) {
            return rootSavePath + "/" + newName.charAt(0) + "/" + newName.charAt(1)+ "/" + newName + "." + imgName.split("\\.")[1];
        } else {
            return rootSavePath + "/" + newName.charAt(0) + "/" + newName.charAt(1)+ "/" + newName;
        }

    }

    public String getMD5(String name){
        String code = null;
        try {
            MessageDigest md  = MessageDigest.getInstance("MD5");
            //byte[] cipherData = md.digest(name.getBytes());
            md.update(name.getBytes());
            code = new BigInteger(1, md.digest()).toString(16);
            //code = cipherData.toString();
        } catch (NoSuchAlgorithmException e) {
            System.out.println("错误：MD5摘要生成出错！");
            e.printStackTrace();
        }
        return code;
    }

    public long getReadCount() {
        return readCount;
    }

    public void setReadCount(long readCount) {
        this.readCount = readCount;
    }

    public String getDataFilePath() {
        return dataFilePath;
    }

    public void setDataFilePath(String dataFilePath) {
        this.dataFilePath = dataFilePath;
    }

    public List<Article> getArticles() {
        return articles;
    }

    public void setArticles(List<Article> articles) {
        this.articles = articles;
    }

    public long getArticleNum() {
        return articleNum;
    }

    public void setArticleNum(long articleNum) {
        this.articleNum = articleNum;
    }

    public long getImgNum() {
        return imgNum;
    }

    public void setImgNum(long imgNum) {
        this.imgNum = imgNum;
    }

    public long getArticleWithImgNum() {
        return articleWithImgNum;
    }

    public void setArticleWithImgNum(long articleWithImgNum) {
        this.articleWithImgNum = articleWithImgNum;
    }

    public long getImgWithSrcNum() {
        return imgWithSrcNum;
    }

    public void setImgWithSrcNum(long imgWithSrcNum) {
        this.imgWithSrcNum = imgWithSrcNum;
    }

    public long getImgWithAltNum() {
        return imgWithAltNum;
    }

    public void setImgWithAltNum(long imgWithAltNum) {
        this.imgWithAltNum = imgWithAltNum;
    }

    public long getImgWithNonEmptySrcNum() {
        return imgWithNonEmptySrcNum;
    }

    public void setImgWithNonEmptySrcNum(long imgWithNonEmptySrcNum) {
        this.imgWithNonEmptySrcNum = imgWithNonEmptySrcNum;
    }

    public long getImgWithNonEmptyAltNum() {
        return imgWithNonEmptyAltNum;
    }

    public void setImgWithNonEmptyAltNum(long imgWithNonEmptyAltNum) {
        this.imgWithNonEmptyAltNum = imgWithNonEmptyAltNum;
    }

    public long getValidImgNum() {
        return validImgNum;
    }

    public void setValidImgNum(long validImgNum) {
        this.validImgNum = validImgNum;
    }

    public String getRootSavePath() {
        return rootSavePath;
    }

    public void setRootSavePath(String rootSavePath) {
        this.rootSavePath = rootSavePath;
    }

    public static void main(String[] args) {
        DataCleansing dc = new DataCleansing();

        Properties prop = new Properties();
        try {
            FileInputStream fis = new FileInputStream("settings.properties");
            prop.load(fis);
            System.out.println("dataFilePath: " + prop.getProperty("dataFilePath"));
            System.out.println("rootFilePath: " + prop.getProperty("rootSavePath"));

        } catch (Exception e) {
            e.printStackTrace();
        }

        dc.setDataFilePath(prop.getProperty("dataFilePath"));
        dc.setRootSavePath(prop.getProperty("rootSavePath"));
        dc.readArticleData();
        System.out.println("articleNum: " + dc.getArticleNum() + " articleWithImgNum: " + dc.getArticleWithImgNum() + " imgNum: " + dc.getImgNum()
                + " imgWithSrcNum: " + dc.getImgWithSrcNum() + " imgWithNonEmptySrcNum: " + dc.getImgWithNonEmptySrcNum()
                + " imgWithAltNum: " + dc.getImgWithAltNum() + " imgWithNonEmptyAltNum: " + dc.getImgWithNonEmptyAltNum()
                + " validImgNum: " + dc.getValidImgNum() + " readCount: " + dc.getReadCount()
        );


        String mongoURI = prop.getProperty("mongoURI");
        String mongoDatabase =  prop.getProperty("mongoDatabase");
        String mongoCollection =  prop.getProperty("mongoCollection");

        MongoClientURI connectionString = new MongoClientURI(mongoURI);
        MongoClient mongoClient= new MongoClient(connectionString);

        CodecRegistry pojoCodecRegistry = fromRegistries(MongoClient.getDefaultCodecRegistry(),
                fromProviders(PojoCodecProvider.builder().automatic(true).build()));

        MongoDatabase database = mongoClient.getDatabase(mongoDatabase).withCodecRegistry(pojoCodecRegistry);


        final MongoCollection<Article> collection = database.getCollection(mongoCollection, Article.class);

        List<Article> articles = dc.getArticles();

        collection.insertMany(articles);


    }

}
